{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 16:Training neural networks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.1 Classifying data with neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.1 Classifying images of handwritten digits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.1 Building the 64-dimensional image vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "digits = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  5., 13.,  9.,  1.,  0.,  0.],\n",
       "       [ 0.,  0., 13., 15., 10., 15.,  5.,  0.],\n",
       "       [ 0.,  3., 15.,  2.,  0., 11.,  8.,  0.],\n",
       "       [ 0.,  4., 12.,  0.,  0.,  8.,  8.,  0.],\n",
       "       [ 0.,  5.,  8.,  0.,  0.,  9.,  8.,  0.],\n",
       "       [ 0.,  4., 11.,  0.,  1., 12.,  7.,  0.],\n",
       "       [ 0.,  2., 14.,  5., 10., 12.,  0.,  0.],\n",
       "       [ 0.,  0.,  6., 13., 10.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.images[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2b926a3da20>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAACtlJREFUeJzt3V9onfUdx/HPZ1HZ/FOsazekqYsBKchgtoaCFITVZdQpuospLShMBr1SlA2s7m53eiPuYghSdYKd0lQFEacTVJywOZO226ypo60dzapryir+GaxUv7vIKXRdtjzp+T1/ztf3C4L5c8jve4jvPs85OXl+jggByOlLbQ8AoD4EDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiZ9XxTZctWxYjIyN1fOtWHTt2rNH1ZmZmGltryZIlja01PDzc2FpDQ0ONrdWkgwcP6ujRo17odrUEPjIyosnJyTq+dasmJiYaXW/Lli2NrTU+Pt7YWvfdd19jay1durSxtZo0NjZW6XacogOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWKXAbW+w/a7tfbbvqXsoAGUsGLjtIUm/kHStpMslbbJ9ed2DAehflSP4Wkn7IuJARByX9JSkG+sdC0AJVQJfIenQKR/P9D4HoOOqBD7fX6z818XUbW+2PWl7cnZ2tv/JAPStSuAzklae8vGwpMOn3ygiHo6IsYgYW758ean5APShSuBvSbrM9qW2z5G0UdJz9Y4FoIQF/x48Ik7Yvl3SS5KGJD0aEXtqnwxA3ypd8CEiXpD0Qs2zACiMV7IBiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kFgtO5tk1eROI5L03nvvNbZWk9syXXTRRY2ttX379sbWkqSbbrqp0fUWwhEcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEisys4mj9o+YvvtJgYCUE6VI/gvJW2oeQ4ANVgw8Ih4XdI/GpgFQGE8BgcSKxY4WxcB3VMscLYuArqHU3QgsSq/JntS0u8krbI9Y/tH9Y8FoIQqe5NtamIQAOVxig4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgO/ddHU1FRjazW5lZAk7d+/v7G1RkdHG1trfHy8sbWa/P9DYusiAA0icCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgsSoXXVxp+1Xb07b32L6zicEA9K/Ka9FPSPpJROy0fYGkKdsvR8Q7Nc8GoE9V9iZ7PyJ29t7/WNK0pBV1Dwagf4t6DG57RNJqSW/O8zW2LgI6pnLgts+X9LSkuyLio9O/ztZFQPdUCtz22ZqLe1tEPFPvSABKqfIsuiU9Imk6Ih6ofyQApVQ5gq+TdKuk9bZ3996+V/NcAAqosjfZG5LcwCwACuOVbEBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kNvB7kx07dqyxtdasWdPYWlKz+4U16corr2x7hC8MjuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGJVLrr4Zdt/sP3H3tZFP2tiMAD9q/JS1X9JWh8Rn/Qun/yG7V9HxO9rng1An6pcdDEkfdL78OzeW9Q5FIAyqm58MGR7t6Qjkl6OCLYuAgZApcAj4rOIuELSsKS1tr85z23YugjomEU9ix4RH0p6TdKGWqYBUFSVZ9GX276w9/5XJH1H0t66BwPQvyrPol8s6XHbQ5r7B2F7RDxf71gASqjyLPqfNLcnOIABwyvZgMQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMrYsWYXx8vLG1MmvyZ7Z06dLG1uoijuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGKVA+9dG32Xba7HBgyIxRzB75Q0XdcgAMqrurPJsKTrJG2tdxwAJVU9gj8o6W5Jn9c4C4DCqmx8cL2kIxExtcDt2JsM6JgqR/B1km6wfVDSU5LW237i9BuxNxnQPQsGHhH3RsRwRIxI2ijplYi4pfbJAPSN34MDiS3qii4R8ZrmdhcFMAA4ggOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQ2MBvXdTk1jRTU//3720GWpPbCU1OTja21s0339zYWl3EERxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSKzSK9l6V1T9WNJnkk5ExFidQwEoYzEvVf12RBytbRIAxXGKDiRWNfCQ9BvbU7Y31zkQgHKqnqKvi4jDtr8m6WXbeyPi9VNv0At/syRdcsklhccEcCYqHcEj4nDvv0ckPStp7Ty3YesioGOqbD54nu0LTr4v6buS3q57MAD9q3KK/nVJz9o+eftfRcSLtU4FoIgFA4+IA5K+1cAsAArj12RAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJDbwWxeNjo42tlaTW+5I0sTERMq1mrRly5a2R2gVR3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwILFKgdu+0PYO23ttT9u+qu7BAPSv6ktVfy7pxYj4ge1zJJ1b40wAClkwcNtLJF0t6YeSFBHHJR2vdywAJVQ5RR+VNCvpMdu7bG/tXR8dQMdVCfwsSWskPRQRqyV9Kume029ke7PtSduTs7OzhccEcCaqBD4jaSYi3ux9vENzwf8Hti4CumfBwCPiA0mHbK/qfeoaSe/UOhWAIqo+i36HpG29Z9APSLqtvpEAlFIp8IjYLWms5lkAFMYr2YDECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxNibbBHuv//+xtaSmt1Xa2ysuRcqTk1NNbbWFx1HcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgsQUDt73K9u5T3j6yfVcTwwHoz4IvVY2IdyVdIUm2hyT9TdKzNc8FoIDFnqJfI2l/RPy1jmEAlLXYwDdKenK+L7B1EdA9lQPvbXpwg6SJ+b7O1kVA9yzmCH6tpJ0R8fe6hgFQ1mIC36T/cXoOoJsqBW77XEnjkp6pdxwAJVXdm+yfkr5a8ywACuOVbEBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4k5ogo/03tWUmL/ZPSZZKOFh+mG7LeN+5Xe74REQv+VVctgZ8J25MR0dwGWQ3Ket+4X93HKTqQGIEDiXUp8IfbHqBGWe8b96vjOvMYHEB5XTqCAyisE4Hb3mD7Xdv7bN/T9jwl2F5p+1Xb07b32L6z7ZlKsj1ke5ft59uepSTbF9reYXtv72d3Vdsz9aP1U/Tetdb/orkrxsxIekvSpoh4p9XB+mT7YkkXR8RO2xdImpL0/UG/XyfZ/rGkMUlLIuL6tucpxfbjkn4bEVt7Fxo9NyI+bHuuM9WFI/haSfsi4kBEHJf0lKQbW56pbxHxfkTs7L3/saRpSSvanaoM28OSrpO0te1ZSrK9RNLVkh6RpIg4PshxS90IfIWkQ6d8PKMkIZxke0TSaklvtjtJMQ9KulvS520PUtiopFlJj/Uefmy1fV7bQ/WjC4F7ns+leWrf9vmSnpZ0V0R81PY8/bJ9vaQjETHV9iw1OEvSGkkPRcRqSZ9KGujnhLoQ+Iyklad8PCzpcEuzFGX7bM3FvS0islyRdp2kG2wf1NzDqfW2n2h3pGJmJM1ExMkzrR2aC35gdSHwtyRdZvvS3pMaGyU91/JMfbNtzT2Wm46IB9qep5SIuDcihiNiRHM/q1ci4paWxyoiIj6QdMj2qt6nrpE00E+KVrpscp0i4oTt2yW9JGlI0qMRsaflsUpYJ+lWSX+2vbv3uZ9GxAstzoSF3SFpW+9gc0DSbS3P05fWf00GoD5dOEUHUBMCBxIjcCAxAgcSI3AgMQIHEiNwIDECBxL7NyyRs2/TGgiSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(digits.images[0], cmap=plt.cm.gray_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FYI, a way to overlay numbers on the pixels showing their brightness values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xt0VOW5x/HvO5OEkHsCAUImMcRoyv0WQIsULwURBauIC+pRETlY6qFQawv1nJ5VTw8e67LFGweCIoWlJQevsDChchGCyC1QrgEEEiEJ4ZILSZgEksy854/ALALRBLP3nrB9Pmu5mhmn7+8ds5/sy8x+H6W1RghhTw5/T0AIYR4pcCFsTApcCBuTAhfCxqTAhbAxKXAhbEwKXAgbkwIXwsakwIWwsQAzBu3YsaNOSkoyY2i/Ki8vtzSvsLDQsqyIiAjLslwul2VZTqfTsiwrffPNN5SUlKjmXmdKgSclJZGTk2PG0H71wQcfWJo3a9Ysy7JGjBhhWdbLL79sWVZ0dLRlWVZKS0tr0evkEF0IG5MCF8LGpMCFsDEpcCFszO8Fvnr1alJTU0lJSTH94ouVWbt372bGjBlMnz6dTz/91NSs6upqCgsLKSgo4Ny5c6ZmFRQU8H//939kZGSwe/duU7MA1q5dy+DBgxk4cCCvvfaaqVl23Bb9WuAej4dnn32WrKwscnNzWbZsGbm5uTd8ltfrZdGiRbzwwgvMnTuXzZs3m/aRl9aa0tJSOnfujMvlwu12U1tba0qW1+vlyy+/5L777mP8+PEcPXrU1I8OPR4Pv/vd71i+fDlbtmzho48+4tChQ6Zl2XFb9GuBb9++nZSUFJKTkwkKCmLChAmsWLHihs86evQoXbp0oXPnzgQEBPDjH/+YHTt2mJJ18eJFAgMDCQwMRClFaGgo1dXVpmSdPXuWyMhIIiIicDqd3HzzzXzzzTemZAHs3LmTbt26kZSURFBQEA8//DBZWVmmZNl1W/RrgRcVFZGQkOB77HK5KCoquuGzysrK6NChg+9xhw4dKCsrMyXL4/E0+jKH0+mkvr7elCy3201oaKjvcWhoKG6325QsgOLiYuLj432Pu3btSnFxsSlZdt0WW1TgSqlRSqnDSqmjSqnZRoU3tR6cUs1+OUeymmGXLLv+zqzMarbAlVJOYB5wH9ADmKiU6mFEuMvloqCgwPe4sLCQrl27GjG0X7M6dOhAaWmp73Fpaalp36hyOp14PB7f46v36Ea6eo/tdrsJCQkxJQsa9thX7tlOnjxJly5dTMmy67bYkj34YOCo1jpPa10LZAAPGhE+aNAgjhw5Qn5+PrW1tWRkZDB27FgjhvZr1s0330xxcTFnzpyhvr6er776qsVfLbxe7dq1o66ujrq6OrTWphZdbGwsFRUVVFZW4vF4OHbsGDfddJMpWQADBgwgLy+P48ePU1tby8cff8yoUaNMybLrttiS76LHAwVXPC4EhhgSHhDAW2+9xb333ovH42Hy5Mn07NnTiKH9muV0Opk8eTJz5szB6/Vy1113NTrnMpJSig4dOnDq1CkAwsPDCQoKMiXL4XAwdOhQsrKy8Hq9pKamEhMTY0oWNPzOXnnlFR555BE8Hg+PPfYY3bt3Ny3Ljtuiam5ddKXUeOBerfWUS48fBwZrradf9bqpwFSAxMTEgcePHzdlwv4kN5sYQ242ab20tDRycnKaPXFvySF6IXDl7scFnLz6RVrrhVrrNK11WmxsbMtnKoQwTUsKfAdwi1Kqm1IqCJgArDR3WkIIIzR7Dq61rldK/RvwD8AJvKu1PmD6zIQQrdaiBR+01plApslzEUIYzO83mwghzCMFLoSNSYELYWNS4ELYmBS4EDYmBS6EjUmBC2FjUuBC2JgpnU3sysqbPwDy8/Mty7KyLZOZd6Bdbfny5ZZlAYwfP97SvObIHlwIG5MCF8LGpMCFsDEpcCFsTApcCBvze4HbsV2M1+ulqKiIoqIiCgsLLb1CbYXPPvuMf/zjH3z++eesXbvW39MxRElJCS+++CK//vWvee6558jMNPfuaKu2Rb9+THa5hcuaNWtwuVwMGjSIsWPH0qOHIasy+y1LKUVcXBwOhwOtNcXFxbRv357g4GDDs/zlzjvvpF27dv6ehmGcTiePP/44ycnJ1NTUMHv2bPr06YPL5TI8y8ptUVoXmUAphcPR8J9Wa93kQveibYmOjiY5ORmA9u3bEx8fb1o3GmlddINnQUNhFxUVceLECdvtvQGys7NZs2YNeXl5/p6K4c6cOUN+fj4pKSmmjG/lttjsIbpS6l3gAeCM1rqXkeF2bRdzeez4+Hg8Hg9nzpyhtrbWtPXKrXb33XfTvn17Lly4QHZ2NuHh4dhlJd0LFy7wl7/8hUmTJpnWQKJNtS4C/gaY0k7Cru1iruR0OgkODqampsb0LKu0b98egODgYFMPZa1WX1/PX/7yF4YNG8aQIYb09mhSm2pdpLXOBkz5Ddq1XYzH4/H1C/N6vdTU1BAYGGhKltXq6+upq6vz/Xz69GkiIyP9PKvW01qzYMEC4uPjeeCBB0zNamuti0xj13YxHo+Hs2fP+g7FQkNDTW3SZ6ULFy7w1VdfAQ1FkZiYaFpDQCsdPnyY7OxsEhMT+e1vfwvAxIkTGTBggOFZbap1EYBSKglY9V3n4D+E1kWXr7Jaxcq7yay8C8rKFlB2vZvMyNZFLSKti4Roe/z+TTYhhHmaLXCl1DJgC5CqlCpUSj1t/rSEEEZoSW+yiVZMRAhhPDlEF8LGpMCFsDEpcCFsTApcCBuTAhfCxqTAhbAxKXAhbEwKXAgbu+FbF+3cudOyLCtv/gA4duyYZVlW3kgzYsQIy7Ks3D5AWhcJISwkBS6EjUmBC2FjUuBC2JgUuBA2JgUuhI35/WOypKQkwsPDcTqdBAQEkJOTY0rOiy++yJdffkl0dLRvna758+ezceNGHA4H0dHR/PGPf7wh1/eeNWsW69evp0OHDqxevRqA//mf/2H9+vUEBgaSmJjIK6+8QkRERKuz5s6dyzvvvINSit69e7N48WLTmjocOHCAkpISgoKCuP322wE4ffo0eXl5uN1uBg8ebMh7AsjMzOTYsWOEhITw9NON1zTZtm0bGzZsYPr06YYtnnnu3DmmTJnC/v37UUrx7rvv+t6jkdrEHvyLL75g9+7dphU3wJgxY3jzzTcbPff444+TkZHB3//+d4YNG8bbb79tWr6Zxo0bx+LFixs9d8cdd5CVlUVmZibdunVj/vz5rc4pKirijTfeICcnh/379+PxeMjIyGj1uN+ma9eu9O/fv9FzYWFh9OnTh6ioKEOzevfu3eRn2JWVlXzzzTeG/SG5bMaMGYwaNYpDhw6xZ88eunfvbuj4l7WJArfCgAEDrvklhYWF+X6uqakxtdOJmQYPHnzNBj9s2DACAhoO0Pr168epU6cMyaqvr6empob6+nqqq6tNbR4RHR19zXryoaGhhIaGGp6VkJDga+hwpXXr1nHXXXcZmlVZWUl2drbvSCEoKMjwP1iX+b3AlVKMHDmSgQMHsnDhQsvz582bx/33309WVha/+MUvLM+3wocffsjw4cNbPU58fDzPP/88iYmJxMXFERkZyciRIw2YYdt05MgRwsPD6dSpk6Hj5uXlERsby1NPPUX//v2ZMmUKbrfb0IzLWrLoYoJS6gul1EGl1AGl1AwjJ7B582Z27dpFVlYW8+bNIzs728jhm/Xss8/y2Wefcd9991m+hrYV5s2bh9Pp5MEHH2z1WOXl5axYsYL8/HxOnjyJ2+3mvffeM2CWbU9dXR1btmxh2LBhho9dX1/Prl27mDZtGv/85z8JDQ01rUd4S/bg9cBvtNbdgduAZ5VShjUyvnyI16lTJx566CG2b99u1NDXZdSoUaxbt84v2Wb56KOP+OKLL5g7d64hpx9r166lW7duxMbGEhgYyMMPP+zrcmI3586do6KignfffZf58+dTVVXF3/72N86fP9/qsV0uFy6Xy9f/7JFHHmHXrl2tHrcpLelNVqy13nXp5yrgIBBvRLjb7aaqqsr38+eff06vXoY2MP1OJ06c8P28ceNGkpKSLMs228aNG1m4cCHp6elNnlt+H4mJiWzdupXq6mq01qxbt860i0P+Fhsby/Tp05k2bRrTpk0jPDycSZMmNbpu83116dKFhIQEDh8+DDSc5/foYdg+s5Hr+pjsUguj/sC2Jv7dla2LWjTe6dOneeihh4CGw5af//znjBplSiNTXnjhBXbu3Mm5c+cYPXo0U6dOZfPmzRw/fhyHw0FcXBy///3vTck224wZM9i2bRvl5eUMHTqUGTNmMH/+fGpra3nyySeBhgtt//3f/92qnCFDhvDII48wYMAAAgIC6N+/P1OnTjXiLTRp3759lJeXU1dXx6ZNm0hOTiYwMJDDhw9TW1vL7t27CQsLM6R/2MqVKzlx4gQ1NTXMmzePO+64g759+xrwLpr25ptv8thjj1FbW0tycvI1n4IYpUW9yQCUUmHARmCO1vrj73ptWlqaNvMjrytZeTtgWlqaZVkgt4saYeDAgZZlAaadS1/N0N5kSqlA4CPg/eaKWwjRdrTkKroCFgEHtdZ/NX9KQgijtGQPPhR4HLhbKbX70j+jTZ6XEMIALelN9iVwY37FS4gfOL9/k00IYR4pcCFsTApcCBuTAhfCxqTAhbAxKXAhbEwKXAgbkwIXwsb8vuhia5WXl1uWZcRdS9fDyhtArGT1DSA/ZLIHF8LGpMCFsDEpcCFsTApcCBuTAhfCxvxa4JMnT6ZTp06NFlr8wx/+QJ8+fejXrx8jR47k5MmThmS9+uqrjB8/nn/913/1Pbd06VImTJjAM888wzPPPMO2bdcsNfe91NbW8vXXX3PgwAFyc3M5c+aMIeN+m9WrV5OamkpKSorpSwZZmQUNa4i//fbbpKens3XrVlOz5s+fz6JFi1i8eDFLliwxLcfK7d6vBT5p0iRfL63Lfvvb37J37152797NAw88wH/9138ZkjVy5Eheeumla54fN24c6enppKen+5axbS2lFC6Xi549e5KamsrZs2epqakxZOyreTwenn32WbKyssjNzWXZsmXk5ube8FkAXq+XNWvWMH78eKZMmUJubi4lJSWm5QFMnDiRp556yrdYpRms3O79WuA/+clPiImJafTcle2F3G63Ye2E+vTpQ3h4uCFjNScwMNDXpM7pdBIcHExdXZ0pWdu3byclJYXk5GSCgoKYMGECK1asuOGzAIqLi4mKiiIqKgqn00n37t05cuSIaXlWsXK7b5NfdPn3f/93li5dSmRkJF988YWpWStWrGDNmjXceuutPPPMM4b/Ebh48SLV1dWm9NOChoaACQkJvscul8uwUw1/ZgFUVVU12vDDw8MpLi42LU8p5etu069fP/r162daVlPM2O5bsuhisFJqu1Jqz6XWRS8akvwd5syZQ0FBAY899hhvvfWWaTljxoxhyZIlLFiwgJiYGNLT0w0d3+PxkJeXh8vlwul0Gjr2ZU0te21WE0Urs/zhscceY9KkSYwfP55du3ZRUFBgab4Z231LDtEvAndrrfsC/YBRSqnbDElvxs9//nM++ugj08aPjo7G6XTicDgYPXq0r9OEEbTW5OXlERMTQ3R0tGHjXs3lcjXaEAsLC03r+GllFjTssSsrK32Pq6qqDOks8l150NDB9NZbbzXsQtf1MnK7b0nrIq21vtyQKfDSPy3rlvA9XHmOtXLlSn70ox+ZFUVpaanv582bNxvWukhrzfHjxwkODqZz586GjPltBg0axJEjR8jPz6e2tpaMjAzGjh17w2cBxMXFUV5ezrlz5/B4PBw8eJCUlBRTsmpra7l48aLv5/z8fGJjY03JaopZ232LzsGVUk5gJ5ACzNNaG9K6aOLEiWzYsIGSkhJcLhcvvvgimZmZHD58GIfDwU033cSCBQta/Ga+y5w5c9i7dy8VFRVMnDiRJ554gj179nDs2DGUUnTu3JmZM2cakuV2uykrKyM4OJiDBw8CDU0WIyMjDRn/SgEBAbz11lvce++9eDweJk+eTM+ePQ3PsToLwOFwMGLECJYvX47Wmt69e5tWdNXV1Xz8cUNPD6/XS48ePUy72cfK7b7FrYsAlFJRwCfAdK31/m97nZWti9auXWtJDsCsWbMsywJr2zJZafbs2f6egmluyNZFl2mtzwEbAHM6BAohDNWSq+ixl/bcKKXaAz8FDpk9MSFE67XkHDwOWHLpPNwBLNdarzJ3WkIII7SkddFeGnqCCyFuMHI3mRA2JgUuhI1JgQthY1LgQtiYFLgQNiYFLoSNSYELYWNS4ELYWJtc0eV6WNm6aMSIEZZl2ZmVvzMz78W/EcgeXAgbkwIXwsakwIWwMSlwIWxMClwIG/N7gb/++uv06tWLnj178tprr5mW87//+79MmTKF3/zmN77nzp8/z5/+9Cd+9atf8ac//Ynz589/xwjXx8qWO3ZtXbRhwwaWLl3KBx98YGoOQGZmJm+++SaLFi3yPVdTU0NGRgYLFy4kIyODCxcuGJLVVOuisrIyRowYwS233MKIESMM+6TBrwW+f/9+3n77bbZv386ePXtYtWqVaZ0r7rzzTl544YVGz3366af07t2bN954g969e/Ppp58akmVlyx07ty5KTU1l9OjRpo1/pd69ezN+/PhGz23dupWkpCSmTp1KUlKSYX+om2pd9PLLL3PPPfdw5MgR7rnnHsP+ePq1wA8ePMhtt91GSEgIAQEBDB8+nE8++cSUrB49elyzpvaOHTsYPnw4AMOHD2fHjh2GZFnZcsfOrYvi4uJo166daeNfKSEhgfbt2zd67ujRo769bK9evQz7HTbVumjFihW+fmhPPvmkYTsbvxZ4r169yM7OprS0lOrqajIzMy3tJlFRUeH7IkR0dHSjRfZbo6mWO0Ye/l+pqXZCRUVFN3xWW+B2u307hbCwMNxut2lZp0+fJi4uDmj4w2ZUR9oWf5Pt0ppsOUCR1voBI8K7d+/OrFmzGDFiBGFhYfTt25eAgBv+y3WWktZF4rtczx58BnDQ6Ak8/fTT7Nq1i+zsbGJiYrjllluMjvhWkZGRvosZ5eXljfa6rWFlyx07ty7yt9DQUN+R1/nz501rIAnQuXNnX2PF4uJiOnXqZMi4LSpwpZQLuB94x5DUK1w+FDlx4gQff/wxEydONDriW6WlpbFx40YANm7cyKBBgwwZ18qWO3ZuXeRvKSkp7N/f0N9j//79pv0OAcaOHcuSJUsAWLJkCQ8++KAh47b0ePg14HeA4Q22x40bR2lpKYGBgcybN8+0mwNee+01cnNzqaqq4he/+AWPPvooP/vZz5g7dy7r16+nY8eOPPfcc4ZkWdlyx86ti9atW8fJkye5cOEC77//PgMHDjStV93KlSs5ceIENTU1zJs3jzvuuIPbbruNFStWsHfvXiIiIgwruqZaF82ePZtHH32URYsWkZiYaNhHg822LlJKPQCM1lr/Uil1J/B8U+fgV/UmG3j8+HFDJtgcKz4jvczqVkJWtcGx2jPPPGNZltV3k92IrYuGAmOVUt8AGcDdSqn3rn6R1nqh1jpNa51mZVdGIcS3a0n74N9rrV1a6yRgArBea/0vps9MCNFqfv+qqhDCPNf1obPWegMN3UWFEDcA2YMLYWNS4ELYmBS4EDYmBS6EjUmBC2FjUuBC2JgUuBA2JgUuhI3d8KsrWHkzgdU3m1jJynZCOTk5lmU9+uijlmW1RbIHF8LGpMCFsDEpcCFsTApcCBuTAhfCxvx6FX3y5MmsWrWKTp06+Ra3M8urr77Ktm3biIqK4u233wZg6dKlZGZmEhkZ6ZvPkCFDDMk7cOAAJSUlBAUFcfvttwMNa1/n5eXhdrsZPHiwYau4zp07l3feeQelFL1792bx4sUEBwcbMvbVFixYwNKlS9Fa88QTTzBt2jRTcgCOHz9ORUUFAQEB9OjRA2hYybWiogKlFO3ateOmm24yZKntDz74gEOHDhEWFsavf/1roKGd0cGDB3E6ncTExDB+/PhrmiN8X6tXr2bGjBl4PB6mTJnC7NmzDRn3an7dgzfVwsUsI0eO5KWXXrrm+XHjxpGenk56erphxQ3QtWtX+vfv3+i5sLAw+vTpQ1RUlGE5RUVFvPHGG+Tk5LB//348Hg8ZGRmGjX+l3Nxcli5dytq1a9m0aROff/45x44dMyULICYm5pqVTCMiIujRowc9evQgODiY06dPG5I1cOBAJk+e3Oi5lJQUZs6cycyZM4mNjWXDhg2GZFnZAsqvBd5UCxez9OnTh/BwwxeF/VbR0dEEBgY2ei40NNSUtbXr6+upqamhvr6e6upq09Yq//rrr0lLS/O1mvrxj3/MZ599ZkoWNKwv73Q6Gz0XERHha7YQGhpKbW2tIVnJycnX7J1vvfVWX35CQgIVFRWGZFnZAuoHfw6+YsUKpk6dyquvvkpVVZW/p3Pd4uPjef7550lMTCQuLo7IyEhGjhxpSlb37t3ZsmULZWVlVFdXs2bNGr+2LiopKTHsNKc5OTk5pKamGjKWlS2gWtr44Bul1D6l1G6llHVfQzLZmDFjWLJkCQsWLCAmJob09HR/T+m6lZeXs2LFCvLz8zl58iRut5v33rtm0VtDpKam8qtf/YqHH36Y8ePH06tXr2v2sFYpLi5GKWXJEeD69etxOBz069fPkPGsbAF1PXvwu7TW/bTWaabMxA+io6NxOp04HA5Gjx7N4cOH/T2l67Z27Vq6detGbGwsgYGBPPzww3z11Vem5T3++ONs2LCBzz77jOjoaG6++WbTsr5NaWkplZWVdOvWzfTeaDt37uTQoUNMmDDBsCwrW0D9oA/RS0tLfT9v3ryZpKQk/03me0pMTGTr1q1UV1ejtWbdunV0797dtLyzZ88CDRvlqlWrGDdunGlZTamoqOD06dMkJyfjcJi7+R4+fJiNGzfyxBNPEBQUZNi4VraAaunnCxr4XCmlgXSt9UIjwptq4fL0008bMfQ15syZw969e6moqGDixIk88cQT7Nmzh2PHjqGUonPnzsycOdOwvH379lFeXk5dXR2bNm0iOTmZwMBADh8+TG1tLbt37yYsLIwBAwa0KmfIkCE88sgjDBgwgICAAPr378/UqVMNehfXevLJJykrKyMwMJBXXnnF0E8Erpafn09VVRX19fXs27ePuLg4Tp8+jdfr5ejRo0DDhbbExMRWZy1btsz3EeZLL73EiBEj2LBhA/X19SxatAho+GP60EMPtTrLyhZQzbYuAlBKddVan1RKdQLWANO11tlXvcYvrYvWrl1rSQ7An//8Z8uyANasWWNZlpV3k/30pz+1LMvqu8lmzZplSY6RrYvQWp+89L9ngE+AwU28RloXCdHGNFvgSqlQpVT45Z+BkYC5XzsTQhiiJefgnYFPLl1BDAD+rrW25utnQohWabbAtdZ5QF8L5iKEMNgP+mMyIexOClwIG5MCF8LGpMCFsDEpcCFsTApcCBuTAhfCxqTAhbCxG751UXJysmVZVrbcgYaFAO2YZSWrbv5oq2QPLoSNSYELYWNS4ELYmBS4EDYmBS6Ejfn9KrpVLVygodFCaGgoTqcTp9Np2mLzABcuXPAtyu90OgkJCTFtBdBVq1axfv16lFIkJCTwy1/+0tBFAq/09ddfk5+fD0BkZCSDBg0ybenkCxcu+LIALl68SNeuXenUqZMpeVZui1Zl+bXAL7dwWbNmDS6Xi0GDBjF27FhfHyozvP/++6avpe31eqmtrSU8PBylFG63m9raWtq1a2d4VllZGVlZWcydO5egoCD++te/8tVXX3HnnXcanlVTU8ORI0cYNWoUTqeTLVu2UFBQYNpqtMHBwb4VYrXW7Nu3z9dHzmhWbotWZvn1EN3KFi5W01o3+sfMJX4v/0HxeDzU1tYSHR1tWpbWGo/Hg9frxePxmNbk8GpVVVW0a9fOlD+SYO22aGWWX/fgTbVw2bZtm2l5SikmTZoENCzZPHHiRFNyHA4HwcHBVFZWopQiICDgmj5lRomJiWHMmDFMmzaNoKAg+vbtS9++5izA0759e1JTU1m1ahVOp5MuXbrQpUsXU7KuVl5ebuofLiu3RSuzWtq6KEop9aFS6pBS6qBS6nYjwq1s4QKwfPlyVq5cybvvvst7773H9u3bTcnxer3U1dURERFBREQEWmvDmuRd7fz58+zYsYN58+aRnp7OhQsXyM7Obv7/+D3U1tZSVFTE/fffz5gxY6ivr8eK5bG9Xi/nzp0z/cjkamZti22xddHrwGqt9Y9oWJ/toBHhVrZwAejcuTMAHTt2ZOTIkezZs8eUnPr6ehwOBw6HA6UUQUFB1NfXm5K1b98+OnXqREREBAEBAQwZMoSvv/7alKzTp08TGhpKu3btcDgcxMfHN+oOY5bKykpCQkJMOwoCa7fFNtW6SCkVAfwEWASgta7VWp8zItzKFi7V1dWcP3/e9/OmTZu49dZbTclyOBzU19f7zr/r6upMu9LcsWNHjhw5wsWLF30XouLj403JCgkJoayszPfezpw5Y0lL5vLyctMvjFq5Lba11kXJwFlgsVKqL7ATmKG1drc63MIWLiUlJUybNg1ouIo5ZswYhg8fbkrW5XPuy+2InU6naR9b3XLLLdx2223MmjULp9NJUlKSaZ1DOnTogMvlYu3atSiliIqKMv1mH6/XS2VlpSHtib6Lldtim2pdpJRKA7YCQ7XW25RSrwOVWus/XPU6v7QuysvLsyQHYODAgZZlASxcaEgLuBax8m6yY8eOWZa1c+dOy7KsZGTrokKgUGt9+TLfh8A1HfOkdZEQbU+zBa61PgUUKKVSLz11D5Br6qyEEIZo6efg04H3lVJBQB7wlHlTEkIYpUUFrrXeDaSZPBchhMHkbjIhbEwKXAgbkwIXwsakwIWwMSlwIWxMClwIG5MCF8LGpMCFsDEpcCFszO+rqraWlb3J/vznP1uWBdb21UpLs+6Lina9w6stkj24EDYmBS6EjUmBC2FjUuBC2JgUuBA25vcCX716NampqaSkpPDyyy/bJmv//v3853/+J//xH//B6tWrTc2qrq6msLCQgoICzp1W8dDCAAAFAUlEQVQzZMHbb3Xq1CmysrLIzMzk0KFDpmaBfbcPq7L8WuCXezRlZWWRm5vLsmXLyM01ZzUoK7O8Xi/Lli1j+vTp/PGPf2THjh2cPHnSlCytNaWlpXTu3BmXy+Xrg2ZW1q5duxg2bBijRo3ixIkTVFZWmpIF9t0+rMz6wfQmszIrPz+fTp06ERsbS0BAAGlpaaY1Wbh48SKBgYEEBgailCI0NJTq6mpTssrKyggLCyMsLAyHw0FCQgJFRUWmZIF9tw8rs/xa4E31aDJrg7Ey6+o2O9HR0aYdOns8nkZNFZxOp2ldVGpqaggJCfE9DgkJoaamxpQssO/2YWVWSzqbpCqldl/xT6VSaqYR4XbtB+VvdvlvaNftw8qsZr+qqrU+DPS7NAknUAR8YkS4XftBRUVFUV5e7ntcXl5OVFSUKVlOpxOPx+N7fPUe3UghISGNDv+rq6tNbR9s1+2jTfUmu8o9wDGttSFtS+zaDyopKYkzZ85QUlJCfX09OTk5prX0bdeuHXV1ddTV1aG1xu12NzqMNlJ0dDTnz5/H7Xbj9XopKCgwtVmkXbePttab7EoTgGVN/YurWhe1LNym/aCcTicTJkzg9ddfx+v1MnToUNMKQSlFhw4dOHXqFADh4eGm9UFzOBz079+f7OxstNZ069aNyMhIU7LAvttHm+pN5nthQ9ODk0BPrfXp73ptWlqazsnJMWB6bYuVvcIA0z+LvZKVd5MtX77csiy7MrI32WX3AbuaK24hRNtxPQU+kW85PBdCtE0tKnClVAgwAvjY3OkIIYzU0t5k1UAHk+cihDCY3282EUKYRwpcCBuTAhfCxqTAhbAxKXAhbEwKXAgbkwIXwsakwIWwsRbfbHJdgyp1FrjeW0o7AiWGT6ZtsOt7k/flPzdprWObe5EpBf59KKVytNbW3dJkIbu+N3lfbZ8cogthY1LgQthYWypwa1dTsJZd35u8rzauzZyDCyGM15b24EIIg7WJAldKjVJKHVZKHVVKzfb3fIyglEpQSn2hlDqolDqglJrh7zkZSSnlVEr9Uym1yt9zMZJSKkop9aFS6tCl393t/p5Ta/j9EP3SWutf07BiTCGwA5iotTanWZNFlFJxQJzWepdSKhzYCfzsRn9flymlngPSgAit9QP+no9RlFJLgE1a63cuLTQaorU2t6OjidrCHnwwcFRrnae1rgUygAf9PKdW01oXa613Xfq5CjgIxPt3VsZQSrmA+4F3/D0XIymlIoCfAIsAtNa1N3JxQ9so8Hig4IrHhdikEC5TSiUB/YFt/p2JYV4Dfgd4/T0RgyUDZ4HFl04/3lFKhfp7Uq3RFgq8qbWdbXNpXykVBnwEzNRam9dr1yJKqQeAM1rrnf6eiwkCgAHAfK11f8AN3NDXhNpCgRcCCVc8dtHQYOGGp5QKpKG439da22VF2qHAWKXUNzScTt2tlHrPv1MyTCFQqLW+fKT1IQ0Ff8NqCwW+A7hFKdXt0kWNCcBKP8+p1VRDu8hFwEGt9V/9PR+jaK1/r7V2aa2TaPhdrdda/4ufp2UIrfUpoEAplXrpqXuAG/qi6PX2JjOc1rpeKfVvwD8AJ/Cu1vqAn6dlhKHA48A+pdTuS8+9oLXO9OOcRPOmA+9f2tnkAU/5eT6t4vePyYQQ5mkLh+hCCJNIgQthY1LgQtiYFLgQNiYFLoSNSYELYWNS4ELYmBS4EDb2/1vZoKOAFofyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(digits.images[0], cmap=plt.cm.gray_r)\n",
    "for i in range(0,8):\n",
    "    for j in range(0,8):\n",
    "        plt.gca().text(i-0.15,j,int(digits.images[0][i][j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  5., 13.,  9.,  1.,  0.,  0.,  0.,  0., 13., 15., 10.,\n",
       "       15.,  5.,  0.,  0.,  3., 15.,  2.,  0., 11.,  8.,  0.,  0.,  4.,\n",
       "       12.,  0.,  0.,  8.,  8.,  0.,  0.,  5.,  8.,  0.,  0.,  9.,  8.,\n",
       "        0.,  0.,  4., 11.,  0.,  1., 12.,  7.,  0.,  0.,  2., 14.,  5.,\n",
       "       10., 12.,  0.,  0.,  0.,  0.,  6., 13., 10.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.matrix.flatten(digits.images[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.33333333, 0.86666667, 0.6       ,\n",
       "       0.06666667, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.86666667, 1.        , 0.66666667, 1.        , 0.33333333,\n",
       "       0.        , 0.        , 0.2       , 1.        , 0.13333333,\n",
       "       0.        , 0.73333333, 0.53333333, 0.        , 0.        ,\n",
       "       0.26666667, 0.8       , 0.        , 0.        , 0.53333333,\n",
       "       0.53333333, 0.        , 0.        , 0.33333333, 0.53333333,\n",
       "       0.        , 0.        , 0.6       , 0.53333333, 0.        ,\n",
       "       0.        , 0.26666667, 0.73333333, 0.        , 0.06666667,\n",
       "       0.8       , 0.46666667, 0.        , 0.        , 0.13333333,\n",
       "       0.93333333, 0.33333333, 0.66666667, 0.8       , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.4       , 0.86666667,\n",
       "       0.66666667, 0.        , 0.        , 0.        ])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.matrix.flatten(digits.images[0]) / 15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.2 Building a random digit classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_classifier(input_vector):\n",
    "    return np.random.rand(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09609494, 0.75660412, 0.78242862, 0.43881297, 0.33470396,\n",
       "       0.92745083, 0.9695724 , 0.12471611, 0.01307655, 0.61261849])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = np.matrix.flatten(digits.images[0]) / 15.\n",
    "result = random_classifier(v)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE** because this is random, you will get a different digit result when you re-run the code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(result).index(max(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.target[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.3 Measuring performance of the digit classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_digit_classify(classifier,test_count=1000):\n",
    "    correct = 0 #<1>\n",
    "    for img, target in zip(digits.images[:test_count], digits.target[:test_count]): #<2>\n",
    "        v = np.matrix.flatten(img) / 15. #<3>\n",
    "        output = classifier(v) #<4>\n",
    "        answer = list(output).index(max(output)) #<5>\n",
    "        if answer == target:\n",
    "            correct += 1 #<6>\n",
    "    return (correct/test_count) #<7>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.089"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(random_classifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Suppose a digit classifier function outputs the following NumPy array.  What digit does it think the image represents?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.00512567e-06, 3.94168539e-05, 5.57124430e-09, 9.31981207e-09,\n",
       "       9.98060276e-01, 9.10328786e-07, 1.56262695e-03, 1.82976466e-04,\n",
       "       1.48519455e-04, 2.54354113e-07])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([5.00512567e-06, 3.94168539e-05, 5.57124430e-09, 9.31981207e-09,\n",
    "       9.98060276e-01, 9.10328786e-07, 1.56262695e-03, 1.82976466e-04,\n",
    "       1.48519455e-04, 2.54354113e-07])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini project:** Find the average of all the images of nines in the data set, in the same way we took averages of images in Chapter 6.  Plot the resulting image. What does it look like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_img(i):\n",
    "    imgs = [img for img,target in zip(digits.images[1000:], digits.target[1000:]) if target==i]\n",
    "    return sum(imgs) / len(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2b926891c50>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAC2hJREFUeJzt3V2IXPUZx/Hfz2wk9SUJNEnVJHYVJBALNRoikiA0tiVG0V70IsEXKoV4oygpiPbK3oukYhAlagVTpY0RRIxWULFCY928tE3cpKRLajbR7q5FjAYbY55e7Aip2TJnM+f8Z/bh+4HFfRn2/wzh65mZPXP+jggByOmsbg8AoDkEDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBifU380jlz5kR/f38Tv/o0Jc/EO3LkSLG1JGlsbKzYWtOnTy+21gUXXFBsrZkzZxZbS5L6+hpJ6jQHDx7U2NiY292ukWn6+/s1MDDQxK8+zYkTJ4qsI0kPPvhgsbUk6fHHHy+21sKFC4uttX79+mJrrV69uthakjRr1qwi61x99dWVbsdDdCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSqxS47VW299s+YPv+pocCUI+2gdueJmmjpOslLZa01vbipgcD0LkqR/Blkg5ExFBEHJf0vKSbmx0LQB2qBD5f0qFTvh5ufQ9Aj6sS+ETvWDntLVy219kesD0wOjra+WQAOlYl8GFJp77VaIGk0943GRFPRMTSiFg6d+7cuuYD0IEqgb8n6TLbl9g+W9IaSS81OxaAOrR9P3hEnLB9l6TXJE2T9FRE7G18MgAdq3TBh4h4RdIrDc8CoGacyQYkRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYmX2WWnQ9u3bi631yCOPFFtLkm6//fZiaw0NDRVb66GHHiq21ooVK4qtJUmzZ88uul47HMGBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcSq7GzylO0R23tKDASgPlWO4L+RtKrhOQA0oG3gEfG2pH8XmAVAzXgODiRWW+BsXQT0ntoCZ+sioPfwEB1IrMqfyZ6T9CdJi2wP2/5582MBqEOVvcnWlhgEQP14iA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYlN+66KjR48WW2vOnDnF1pKkRYsWFVvr2LFjxdbatm1bsbU+/vjjYmtJUn9/f9H12uEIDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYlUuurjQ9pu2B23vtX1PicEAdK7KuegnJP0iInbaPl/SDtuvR8T7Dc8GoENV9ib7MCJ2tj4/KmlQ0vymBwPQuUk9B7fdL2mJpHcn+BlbFwE9pnLgts+T9IKkeyPi02/+nK2LgN5TKXDb0zUe9+aI2NrsSADqUuVVdEt6UtJgRDzc/EgA6lLlCL5c0m2SVtre3fpY3fBcAGpQZW+ydyS5wCwAasaZbEBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kNuX3Jlu8eHGxtS6//PJia0nSo48+WmytL7/8stha8+bNK7bW4cOHi60lSVdddVXR9drhCA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJFblooszbP/Z9l9aWxf9qsRgADpX5VTV/0haGRGftS6f/I7tbRGxveHZAHSoykUXQ9JnrS+ntz6iyaEA1KPqxgfTbO+WNCLp9Yhg6yJgCqgUeER8FRFXSFogaZnt701wG7YuAnrMpF5Fj4hPJL0laVUj0wCoVZVX0efant36/FuSfihpX9ODAehclVfRL5T0jO1pGv8fwu8i4uVmxwJQhyqvov9V43uCA5hiOJMNSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcSm/NZFF198cbG1NmzYUGwtSdq1a1extcbfFVzGxo0bi601MjJSbK1exBEcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiscuCta6Pvss312IApYjJH8HskDTY1CID6Vd3ZZIGkGyRtanYcAHWqegTfIOk+SScbnAVAzapsfHCjpJGI2NHmduxNBvSYKkfw5ZJusn1Q0vOSVtp+9ps3Ym8yoPe0DTwiHoiIBRHRL2mNpDci4tbGJwPQMf4ODiQ2qSu6RMRbGt9dFMAUwBEcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcSm/NZFtoutVXKbJEmaP39+sbU++OCDYmvNmjWr2FpffPFFsbV6EUdwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCCxSmeyta6oelTSV5JORMTSJocCUI/JnKr6g4gYa2wSALXjITqQWNXAQ9IfbO+wva7JgQDUp+pD9OURccT2PEmv294XEW+feoNW+Ouk8u+6AjCxSkfwiDjS+u+IpBclLZvgNmxdBPSYKpsPnmv7/K8/l/RjSXuaHgxA56o8RP+OpBdbF1bok/TbiHi10akA1KJt4BExJOn7BWYBUDP+TAYkRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYlN+66Ljx48XW2vr1q3F1pKkiy66qNha+/fvL7bWyMhIsbXmzZtXbC1JOnnyZNH12uEIDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kVilw27Ntb7G9z/ag7WuaHgxA56qeqvprSa9GxE9tny3pnAZnAlCTtoHbninpWkk/k6SIOC6p3AngAM5YlYfol0oalfS07V22N7Wujw6gx1UJvE/SlZIei4glkj6XdP83b2R7ne0B2wOjo6M1jwngTFQJfFjScES82/p6i8aD/x9sXQT0nraBR8RHkg7ZXtT61nWS3m90KgC1qPoq+t2SNrdeQR+SdEdzIwGoS6XAI2K3pKUNzwKgZpzJBiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kNuX3JuvrK3cX9uzZU2wtSbrzzjuLrTVjxoxia91yyy3F1lqxYkWxtSTprLN665jZW9MAqBWBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJBY28BtL7K9+5SPT23fW2I4AJ1pe55nROyXdIUk2Z4m6bCkFxueC0ANJvsQ/TpJ/4iIfzYxDIB6TTbwNZKem+gHbF0E9J7Kgbc2PbhJ0u8n+jlbFwG9ZzJH8Osl7YyIfzU1DIB6TSbwtfo/D88B9KZKgds+R9KPJG1tdhwAdaq6N9kxSd9ueBYANeNMNiAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSc0TU/0vtUUmTfUvpHEljtQ/TG7LeN+5X93w3Itq+q6uRwM+E7YGIWNrtOZqQ9b5xv3ofD9GBxAgcSKyXAn+i2wM0KOt94371uJ55Dg6gfr10BAdQs54I3PYq2/ttH7B9f7fnqYPthbbftD1oe6/te7o9U51sT7O9y/bL3Z6lTrZn295ie1/r3+6abs/Uia4/RG9da/3vGr9izLCk9yStjYj3uzpYh2xfKOnCiNhp+3xJOyT9ZKrfr6/ZXi9pqaSZEXFjt+epi+1nJP0xIja1LjR6TkR80u25zlQvHMGXSToQEUMRcVzS85Ju7vJMHYuIDyNiZ+vzo5IGJc3v7lT1sL1A0g2SNnV7ljrZninpWklPSlJEHJ/KcUu9Efh8SYdO+XpYSUL4mu1+SUskvdvdSWqzQdJ9kk52e5CaXSppVNLTracfm2yf2+2hOtELgXuC76V5ad/2eZJekHRvRHza7Xk6ZftGSSMRsaPbszSgT9KVkh6LiCWSPpc0pV8T6oXAhyUtPOXrBZKOdGmWWtmervG4N0dElivSLpd0k+2DGn86tdL2s90dqTbDkoYj4utHWls0HvyU1QuBvyfpMtuXtF7UWCPppS7P1DHb1vhzucGIeLjb89QlIh6IiAUR0a/xf6s3IuLWLo9Vi4j4SNIh24ta37pO0pR+UbTSZZObFBEnbN8l6TVJ0yQ9FRF7uzxWHZZLuk3S32zvbn3vlxHxShdnQnt3S9rcOtgMSbqjy/N0pOt/JgPQnF54iA6gIQQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJPZfky/Fz5M++BYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(average_img(9), cmap=plt.cm.gray_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini project:** Build a better classifier than a random one by finding the average image of each kind of digit in the test data set, and comparing a target image with all of the averages.  Specifically, return a vector of the dot products of the target image with each average digit image.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_digits = [np.matrix.flatten(average_img(i)) for i in range(10)]\n",
    "def compare_to_avg(v):\n",
    "    return [np.dot(v,avg_digits[i]) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.853"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(compare_to_avg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.2 Designing a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.1 Organizing neurons and connections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.2 Data flow through a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.3 Calculating activations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.4 Calculating activations in matrix notation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.5 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.3 Building a neural network in Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.1 Implementing an MLP class in Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP():\n",
    "    def __init__(self,layer_sizes): #<1>\n",
    "        self.layer_sizes = layer_sizes\n",
    "        self.weights = [\n",
    "            np.random.rand(n,m) #<2>\n",
    "            for m,n in zip(layer_sizes[:-1],layer_sizes[1:]) #<3>\n",
    "        ]\n",
    "        self.biases = [np.random.rand(n) for n in layer_sizes[1:]] #<4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([2,3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE** these numbers are randomly initialized, so your results below will vary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0.04098182, 0.61121605],\n",
       "        [0.99402839, 0.43278994],\n",
       "        [0.14826355, 0.39370373]])]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.62871743, 0.55151304, 0.91445791])]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.biases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.2 Evaluating the MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp\n",
    "def sigmoid(x):\n",
    "    return 1 / (1+exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP():\n",
    "    def __init__(self,layer_sizes): #<1>\n",
    "        self.layer_sizes = layer_sizes\n",
    "        self.weights = [\n",
    "            np.random.rand(n,m) #<2>\n",
    "            for m,n in zip(layer_sizes[:-1],layer_sizes[1:]) #<3>\n",
    "        ]\n",
    "        self.biases = [np.random.rand(n) for n in layer_sizes[1:]] #<4>\n",
    "    def feedforward(self,v):\n",
    "        activations = [] #<1>\n",
    "        a = v\n",
    "        activations.append(a) #<2>\n",
    "        for w,b in zip(self.weights, self.biases): #<3>\n",
    "            z = w @ a + b #<4>\n",
    "            a = [sigmoid(x) for x in z] #<5>\n",
    "            activations.append(a) #<6>\n",
    "        return activations\n",
    "    def evaluate(self,v):\n",
    "        return np.array(self.feedforward(v)[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.3 Testing the classification performance of an MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([64,16,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.matrix.flatten(digits.images[0]) / 15."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.99996588, 0.99829151, 0.99979785, 0.9998958 , 0.99991507,\n",
       "       0.99982444, 0.9999176 , 0.99863889, 0.99977906, 0.99887847])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.evaluate(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.099"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(nn.evaluate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.4 Training a neural network using gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.1 Framing training as a minimization problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.2 Calculating gradients with backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.3 Automatic training with scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([np.matrix.flatten(img) for img in digits.images[:1000]]) / 15.0\n",
    "y = digits.target[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(16,), #<1>\n",
    "                    activation='logistic', #<2>\n",
    "                    max_iter=100, #<3>\n",
    "                    verbose=10, #<4>\n",
    "                    random_state=1, #<5>\n",
    "                    learning_rate_init=.1) #<6>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 2.21958598\n",
      "Iteration 2, loss = 1.56912978\n",
      "Iteration 3, loss = 0.98970277\n",
      "Iteration 4, loss = 0.57473464\n",
      "Iteration 5, loss = 0.34048448\n",
      "Iteration 6, loss = 0.21495855\n",
      "Iteration 7, loss = 0.14366771\n",
      "Iteration 8, loss = 0.11077020\n",
      "Iteration 9, loss = 0.08764273\n",
      "Iteration 10, loss = 0.07193546\n",
      "Iteration 11, loss = 0.06020348\n",
      "Iteration 12, loss = 0.04961899\n",
      "Iteration 13, loss = 0.03979645\n",
      "Iteration 14, loss = 0.03334502\n",
      "Iteration 15, loss = 0.02996006\n",
      "Iteration 16, loss = 0.02603968\n",
      "Iteration 17, loss = 0.02355514\n",
      "Iteration 18, loss = 0.02137348\n",
      "Iteration 19, loss = 0.01967878\n",
      "Iteration 20, loss = 0.01751214\n",
      "Iteration 21, loss = 0.01617330\n",
      "Iteration 22, loss = 0.01460386\n",
      "Iteration 23, loss = 0.01408517\n",
      "Iteration 24, loss = 0.01270504\n",
      "Iteration 25, loss = 0.01191634\n",
      "Iteration 26, loss = 0.01114222\n",
      "Iteration 27, loss = 0.01045989\n",
      "Iteration 28, loss = 0.00983648\n",
      "Iteration 29, loss = 0.00920912\n",
      "Iteration 30, loss = 0.00890851\n",
      "Iteration 31, loss = 0.00843426\n",
      "Iteration 32, loss = 0.00796039\n",
      "Iteration 33, loss = 0.00749839\n",
      "Iteration 34, loss = 0.00726271\n",
      "Iteration 35, loss = 0.00673963\n",
      "Iteration 36, loss = 0.00655405\n",
      "Iteration 37, loss = 0.00626207\n",
      "Iteration 38, loss = 0.00600639\n",
      "Iteration 39, loss = 0.00581857\n",
      "Iteration 40, loss = 0.00557529\n",
      "Iteration 41, loss = 0.00533573\n",
      "Iteration 42, loss = 0.00519479\n",
      "Iteration 43, loss = 0.00505128\n",
      "Iteration 44, loss = 0.00490121\n",
      "Iteration 45, loss = 0.00469161\n",
      "Iteration 46, loss = 0.00459590\n",
      "Iteration 47, loss = 0.00464844\n",
      "Iteration 48, loss = 0.00445157\n",
      "Iteration 49, loss = 0.00425515\n",
      "Iteration 50, loss = 0.00424934\n",
      "Iteration 51, loss = 0.00397800\n",
      "Iteration 52, loss = 0.00399927\n",
      "Iteration 53, loss = 0.00383932\n",
      "Iteration 54, loss = 0.00372439\n",
      "Iteration 55, loss = 0.00361744\n",
      "Iteration 56, loss = 0.00356447\n",
      "Iteration 57, loss = 0.00345899\n",
      "Iteration 58, loss = 0.00336792\n",
      "Iteration 59, loss = 0.00330330\n",
      "Iteration 60, loss = 0.00321734\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='logistic', alpha=0.0001, batch_size='auto',\n",
       "       beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(16,), learning_rate='constant',\n",
       "       learning_rate_init=0.1, max_iter=100, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=1, shuffle=True,\n",
       "       solver='adam', tol=0.0001, validation_fraction=0.1, verbose=10,\n",
       "       warm_start=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.fit(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.99766643e-01, 8.43331208e-11, 3.47867059e-06, 1.49956270e-07,\n",
       "       1.88677660e-06, 3.44652605e-05, 6.23829017e-06, 1.09043503e-04,\n",
       "       1.11195821e-07, 7.79837557e-05])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp._predict(x)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sklearn_trained_classify(v):\n",
    "    return mlp._predict([v])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(sklearn_trained_classify)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Modify the `test_digit_classify` function to work on a custom range of examples in the test set.  How does it do on the next 500 examples after the 1,000 training examples?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_digit_classify(classifier,start=0,test_count=1000):\n",
    "    correct = 0\n",
    "    end = start + test_count #<1>\n",
    "    for img, target in zip(digits.images[start:end], digits.target[start:end]): #<2>\n",
    "        v = np.matrix.flatten(img) / 15.\n",
    "        output = classifier(v)\n",
    "        answer = list(output).index(max(output))\n",
    "        if answer == target:\n",
    "            correct += 1\n",
    "    return (correct/test_count) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.962"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(sklearn_trained_classify,start=1000,test_count=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Using the squared distance cost function, what is the cost of your randomly-generated MLP for the first 1,000 training examples?  What is the cost of the scikit-learn MLP?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def y_vec(digit):\n",
    "    return np.array([1 if i == digit else 0 for i in range(0,10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_one(classifier,x,i):\n",
    "    return sum([(classifier(x)[j] - y_vec(i)[j])**2 for j in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_cost(classifier):\n",
    "    return sum([cost_one(classifier,x[j],y[j]) for j in range(1000)])/1000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.990834701722013"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_cost(nn.evaluate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.670512721637246e-05"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_cost(sklearn_trained_classify)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** Extract the MLPClassifier weights and biases using its properties called `coefs_` and `intercepts_`, respectively.  Plug these weights and biases into the MLP class we built from scratch and show that your resulting MLP performs well on digit classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([64,16,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.weights = [w.T for w in mlp.coefs_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.biases = mlp.intercepts_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.962"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(nn.evaluate,start=1000,test_count=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.5 Calculating gradients with backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.1 Finding the cost in terms of the last layer weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.1 Finding the cost in terms of the last layer weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.2 Calculating the partial derivatives for the last layer weights using the chain rule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.3 Exercises "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** Use SymPy or your own code from chapter 10 to automatically find the derivative of the sigmoid function:\n",
    "\n",
    "$$\\sigma(x) = \\frac{1}{1+e^{-x}}$$\n",
    "\n",
    "Show that the answer you get is equal to $\\sigma(x)(1-\\sigma(x))$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exp(-x)/(1 + exp(-x))**2"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sympy import *\n",
    "X = symbols('x')\n",
    "diff(1 / (1+exp(-X)),X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
